
import urllib
import webapp2
from cgi import parse_qs, escape

from google.appengine.ext import db
from google.appengine.api import users
from google.appengine.api import urlfetch

import jinja2
import os, re

import sys
sys.path.insert(0, 'libs')

from bs4 import BeautifulSoup

JINJA_ENVIRONMENT = jinja2.Environment(
    loader=jinja2.FileSystemLoader(os.path.dirname(__file__)))

DISTRICT_MAPPING = {
    '95035': 152,
    '95134': 152,
    '95110': 152,
    '78108': 15,
    '94012': 15,
    '94103': 15,
    }

def guestbook_key(guestbook_name=None):
    """Constructs a Datastore key for a Guestbook entity with guestbook_name."""
    return db.Key.from_path('Guestbook', guestbook_name or 'default_guestbook')

# define the local database
class Representative(db.Model):
    """Models an individual Guestbook entry with author, content, and date."""
    zipcode = db.StringProperty()
    name = db.StringProperty()
    href = db.StringProperty()
    homelessNumber = db.IntegerProperty()


class AdvocatePage(webapp2.RequestHandler):

    def validate(self, address):
        address = address.strip()
        if not address:
            return ""
        # try to validate the zip code, for 5 digits only
        postal_code = re.search(r'(\d{5}(\-\d{4})?)', address)
        # print "Got address '%s', and postal_code: %r" % (address, postal_code)

        if not postal_code:
            return ""
        if len(postal_code.groups()) != 2:
            return ""
        zipcode = postal_code.groups()[0]
        if not re.match('^\d{5}$', zipcode):
            return ""

        return zipcode

    # get method is for browser default access
    def get(self):
        zipcode = escape(self.request.get('zipcode'))
        zipcode = self.validate(zipcode)
        homelessNumber = 0
        results = {}
        if zipcode:
            results = Representative.all().ancestor(
                guestbook_key("representatives")).filter("zipcode = ", zipcode)
            if results.count() >= 1:
                homelessNumber = results[0].homelessNumber

        template_values = {
            'zipcode': zipcode,
            'results': results,
            'homelessNumber': homelessNumber,
        }

        template = JINJA_ENVIRONMENT.get_template('advocate.html')
        self.response.write(template.render(template_values))

    # post is requested after user click submit
    def post(self):
        address = escape(self.request.get('address'))
        zipcode = self.validate(address)

        if zipcode:
            results = Representative.all().ancestor(
                guestbook_key("representatives")).filter(
                "zipcode = ", zipcode)
            if results.count() == 0:
                # the crawler consumes urlfetch quota in GAE, and is slowing process
                # only call it when it's not cached in local datastore
                self.crawl(zipcode)

        query_params = {'zipcode': zipcode}
        self.redirect('/advocate?' + urllib.urlencode(query_params))

    def crawl(self, zipcode):
        # try to retrieve the homeless people count
        homelessNumber = 0
        sitecode = DISTRICT_MAPPING.get(zipcode, 15)
        url = "http://www.hudhdx.info/publicAharReports.aspx?site=%d&year=2011&rpts=LocalReports&type=html" % sitecode
        try:
            result = urlfetch.fetch(url, deadline=20)
        except DownloadError:
            pass
        else:
            if result.status_code == 200:
                homelessNumber = self.get_homeless_count(result.content)

        # if not found in database, we do a hard query through web crawling
        ur = urllib.urlopen('http://house.gov/htbin/findrep?ZIP=' + zipcode)
        soup = BeautifulSoup(ur.read())
        ur.close()
        items = soup.find_all("p", { "class" : "rep" })

        # if someone else has insert entries during urlfetch, we just skip
        results = Representative.all().ancestor(
            guestbook_key("representatives")).filter(
            "zipcode = ", zipcode)
        if results.count() > 0:
            return

        for item in items:
            rep = Representative(parent=guestbook_key("representatives"))
            rep.zipcode = zipcode
            rep.name = item.a.text.strip()
            rep.href = item.a.attrs['href']
            # print "Put in a recent search:", item, rep.zipcode, rep.name
            rep.homelessNumber = homelessNumber
            rep.put()

    def get_homeless_count(self, content):
        soup = BeautifulSoup(content)
        item = soup.find('tr', class_='bottomBorderRow')
        num = list(item.children)[-2].text.strip()
        if num.find(',') > 0:
            num = num.replace(',', '')
        return int(num)

# Here in the main application, we define the default index, advocate and query page
app = webapp2.WSGIApplication([('/', AdvocatePage),
                               ('/advocate', AdvocatePage),
                               ('/query', AdvocatePage)],
                              debug=True)

